From d1be12dd766ed7ed8a6f5fb77708c1b7a53e3092 Mon Sep 17 00:00:00 2001 From: "kaf24@firebug.cl.cam.ac.uk" Date: Fri, 3 Jun 2005 16:42:10 +0000 Subject: [PATCH] bitkeeper revision 1.1664.1.1 (42a08862ToP8uoeBgUzDwAQBMXo4wg) Event-channel CPU affinity. Currently all event channels still bind to VCPU#0 at start of day, and have their binding automatically changed when bound to a VIRQ or IPI source. XenLinux maintains a per-cpu evtchn mask denoting which event channels are bound to each cpu. Todo: Allow guests to change binding of of non-ipi and non-virq evtchns. Signed-off-by: Keir Fraser --- .../arch/xen/kernel/evtchn.c | 47 ++- xen/arch/x86/irq.c | 14 +- xen/common/domain.c | 6 +- xen/common/event_channel.c | 337 +++++++++--------- xen/include/public/event_channel.h | 34 +- xen/include/xen/event.h | 7 +- xen/include/xen/sched.h | 34 +- 7 files changed, 251 insertions(+), 228 deletions(-) diff --git a/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c index c88b802268..6d8deeaec2 100644 --- a/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c +++ b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c @@ -74,6 +74,33 @@ static int irq_bindcount[NR_IRQS]; /* Bitmap indicating which PIRQs require Xen to be notified on unmask. */ static unsigned long pirq_needs_unmask_notify[NR_PIRQS/sizeof(unsigned long)]; +#ifdef CONFIG_SMP + +static u8 cpu_evtchn[NR_EVENT_CHANNELS]; +static u32 cpu_evtchn_mask[NR_CPUS][NR_EVENT_CHANNELS/32]; + +#define active_evtchns(cpu,sh,idx) \ + ((sh)->evtchn_pending[idx] & \ + cpu_evtchn_mask[cpu][idx] & \ + ~(sh)->evtchn_mask[idx]) + +static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu) +{ + clear_bit(chn, (unsigned long *)cpu_evtchn_mask[cpu_evtchn[chn]]); + set_bit(chn, (unsigned long *)cpu_evtchn_mask[cpu]); + cpu_evtchn[chn] = cpu; +} + +#else + +#define active_evtchns(cpu,sh,idx) \ + ((sh)->evtchn_pending[idx] & \ + ~(sh)->evtchn_mask[idx]) + +#define bind_evtchn_to_cpu(chn,cpu) ((void)0) + +#endif + /* Upcall to generic IRQ layer. */ #ifdef CONFIG_X86 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,9) @@ -109,9 +136,9 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs) { u32 l1, l2; unsigned int l1i, l2i, port; - int irq; + int irq, cpu = smp_processor_id(); shared_info_t *s = HYPERVISOR_shared_info; - vcpu_info_t *vcpu_info = &s->vcpu_data[smp_processor_id()]; + vcpu_info_t *vcpu_info = &s->vcpu_data[cpu]; vcpu_info->evtchn_upcall_pending = 0; @@ -122,7 +149,7 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs) l1i = __ffs(l1); l1 &= ~(1 << l1i); - while ( (l2 = s->evtchn_pending[l1i] & ~s->evtchn_mask[l1i]) != 0 ) + while ( (l2 = active_evtchns(cpu, s, l1i)) != 0 ) { l2i = __ffs(l2); l2 &= ~(1 << l2i); @@ -171,6 +198,8 @@ int bind_virq_to_irq(int virq) irq_to_evtchn[irq] = evtchn; per_cpu(virq_to_irq, cpu)[virq] = irq; + + bind_evtchn_to_cpu(evtchn, cpu); } irq_bindcount[irq]++; @@ -225,8 +254,13 @@ int bind_ipi_on_cpu_to_irq(int cpu, int ipi) irq_to_evtchn[irq] = evtchn; per_cpu(ipi_to_evtchn, cpu)[ipi] = evtchn; - } else + + bind_evtchn_to_cpu(evtchn, cpu); + } + else + { irq = evtchn_to_irq[evtchn]; + } irq_bindcount[irq]++; @@ -546,6 +580,11 @@ void __init init_IRQ(void) spin_lock_init(&irq_mapping_update_lock); +#ifdef CONFIG_SMP + /* By default all event channels notify CPU#0. */ + memset(cpu_evtchn_mask[0], ~0, sizeof(cpu_evtchn_mask[0])); +#endif + for ( cpu = 0; cpu < NR_CPUS; cpu++ ) { /* No VIRQ -> IRQ mappings. */ for ( i = 0; i < NR_VIRQS; i++ ) diff --git a/xen/arch/x86/irq.c b/xen/arch/x86/irq.c index a5657e72f8..fe9633972c 100644 --- a/xen/arch/x86/irq.c +++ b/xen/arch/x86/irq.c @@ -184,22 +184,22 @@ typedef struct { u8 nr_guests; u8 in_flight; u8 shareable; - struct vcpu *guest[IRQ_MAX_GUESTS]; + struct domain *guest[IRQ_MAX_GUESTS]; } irq_guest_action_t; static void __do_IRQ_guest(int irq) { irq_desc_t *desc = &irq_desc[irq]; irq_guest_action_t *action = (irq_guest_action_t *)desc->action; - struct vcpu *v; + struct domain *d; int i; for ( i = 0; i < action->nr_guests; i++ ) { - v = action->guest[i]; - if ( !test_and_set_bit(irq, &v->domain->pirq_mask) ) + d = action->guest[i]; + if ( !test_and_set_bit(irq, &d->pirq_mask) ) action->in_flight++; - send_guest_pirq(v, irq); + send_guest_pirq(d, irq); } } @@ -294,7 +294,7 @@ int pirq_guest_bind(struct vcpu *v, int irq, int will_share) goto out; } - action->guest[action->nr_guests++] = v; + action->guest[action->nr_guests++] = v->domain; out: spin_unlock_irqrestore(&desc->lock, flags); @@ -328,7 +328,7 @@ int pirq_guest_unbind(struct domain *d, int irq) else { i = 0; - while ( action->guest[i] && action->guest[i]->domain != d ) + while ( action->guest[i] && (action->guest[i] != d) ) i++; memmove(&action->guest[i], &action->guest[i+1], IRQ_MAX_GUESTS-i-1); action->nr_guests--; diff --git a/xen/common/domain.c b/xen/common/domain.c index a3c42c1464..47fca238fc 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -54,9 +54,9 @@ struct domain *do_createdomain(domid_t dom_id, unsigned int cpu) set_bit(_DOMF_idle_domain, &d->domain_flags); if ( !is_idle_task(d) && - ((init_event_channels(d) != 0) || (grant_table_create(d) != 0)) ) + ((evtchn_init(d) != 0) || (grant_table_create(d) != 0)) ) { - destroy_event_channels(d); + evtchn_destroy(d); free_domain_struct(d); return NULL; } @@ -251,7 +251,7 @@ void domain_destruct(struct domain *d) *pd = d->next_in_hashbucket; write_unlock(&domlist_lock); - destroy_event_channels(d); + evtchn_destroy(d); grant_table_destroy(d); free_perdomain_pt(d); diff --git a/xen/common/event_channel.c b/xen/common/event_channel.c index d2f0c47452..75353209f9 100644 --- a/xen/common/event_channel.c +++ b/xen/common/event_channel.c @@ -27,50 +27,31 @@ #include #include -#define INIT_EVENT_CHANNELS 16 -#define MAX_EVENT_CHANNELS 1024 -#define EVENT_CHANNELS_SPREAD 32 - - -static int get_free_port(struct vcpu *v) +#define bucket_from_port(d,p) \ + ((d)->evtchn[(p)/EVTCHNS_PER_BUCKET]) +#define port_is_valid(d,p) \ + (((p) >= 0) && ((p) < MAX_EVTCHNS) && \ + (bucket_from_port(d,p) != NULL)) +#define evtchn_from_port(d,p) \ + (&(bucket_from_port(d,p))[(p)&(EVTCHNS_PER_BUCKET-1)]) + +static int get_free_port(struct domain *d) { - struct domain *d = v->domain; - int max, port; - event_channel_t *chn; - - max = d->max_event_channel; - chn = d->event_channel; - - for ( port = v->vcpu_id * EVENT_CHANNELS_SPREAD; port < max; port++ ) - if ( chn[port].state == ECS_FREE ) - break; + struct evtchn *chn; + int port; - if ( port >= max ) - { - if ( max == MAX_EVENT_CHANNELS ) - return -ENOSPC; + for ( port = 0; port_is_valid(d, port); port++ ) + if ( evtchn_from_port(d, port)->state == ECS_FREE ) + return port; - if ( port == 0 ) - max = INIT_EVENT_CHANNELS; - else - max = port + EVENT_CHANNELS_SPREAD; - - chn = xmalloc_array(event_channel_t, max); - if ( unlikely(chn == NULL) ) - return -ENOMEM; + if ( port == MAX_EVTCHNS ) + return -ENOSPC; - memset(chn, 0, max * sizeof(event_channel_t)); - - if ( d->event_channel != NULL ) - { - memcpy(chn, d->event_channel, d->max_event_channel * - sizeof(event_channel_t)); - xfree(d->event_channel); - } - - d->event_channel = chn; - d->max_event_channel = max; - } + chn = xmalloc_array(struct evtchn, EVTCHNS_PER_BUCKET); + if ( unlikely(chn == NULL) ) + return -ENOMEM; + memset(chn, 0, EVTCHNS_PER_BUCKET * sizeof(*chn)); + bucket_from_port(d, port) = chn; return port; } @@ -78,18 +59,20 @@ static int get_free_port(struct vcpu *v) static long evtchn_alloc_unbound(evtchn_alloc_unbound_t *alloc) { + struct evtchn *chn; struct domain *d = current->domain; int port; - spin_lock(&d->event_channel_lock); + spin_lock(&d->evtchn_lock); - if ( (port = get_free_port(current)) >= 0 ) + if ( (port = get_free_port(d)) >= 0 ) { - d->event_channel[port].state = ECS_UNBOUND; - d->event_channel[port].u.unbound.remote_domid = alloc->dom; + chn = evtchn_from_port(d, port); + chn->state = ECS_UNBOUND; + chn->u.unbound.remote_domid = alloc->dom; } - spin_unlock(&d->event_channel_lock); + spin_unlock(&d->evtchn_lock); if ( port < 0 ) return port; @@ -102,8 +85,8 @@ static long evtchn_alloc_unbound(evtchn_alloc_unbound_t *alloc) static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) { #define ERROR_EXIT(_errno) do { rc = (_errno); goto out; } while ( 0 ) + struct evtchn *chn1, *chn2; struct domain *d1, *d2; - struct vcpu *v1, *v2; int port1 = bind->port1, port2 = bind->port2; domid_t dom1 = bind->dom1, dom2 = bind->dom2; long rc = 0; @@ -111,9 +94,6 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) if ( !IS_PRIV(current->domain) && (dom1 != DOMID_SELF) ) return -EPERM; - if ( (port1 < 0) || (port2 < 0) ) - return -EINVAL; - if ( dom1 == DOMID_SELF ) dom1 = current->domain->domain_id; if ( dom2 == DOMID_SELF ) @@ -127,63 +107,61 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) return -ESRCH; } - v1 = d1->vcpu[0]; /* XXX */ - v2 = d2->vcpu[0]; /* XXX */ - /* Avoid deadlock by first acquiring lock of domain with smaller id. */ if ( d1 < d2 ) { - spin_lock(&d1->event_channel_lock); - spin_lock(&d2->event_channel_lock); + spin_lock(&d1->evtchn_lock); + spin_lock(&d2->evtchn_lock); } else { if ( d1 != d2 ) - spin_lock(&d2->event_channel_lock); - spin_lock(&d1->event_channel_lock); + spin_lock(&d2->evtchn_lock); + spin_lock(&d1->evtchn_lock); } /* Obtain, or ensure that we already have, a valid . */ if ( port1 == 0 ) { - if ( (port1 = get_free_port(v1)) < 0 ) + if ( (port1 = get_free_port(d1)) < 0 ) ERROR_EXIT(port1); } - else if ( port1 >= d1->max_event_channel ) + else if ( !port_is_valid(d1, port1) ) ERROR_EXIT(-EINVAL); + chn1 = evtchn_from_port(d1, port1); /* Obtain, or ensure that we already have, a valid . */ if ( port2 == 0 ) { /* Make port1 non-free while we allocate port2 (in case dom1==dom2). */ - u16 tmp = d1->event_channel[port1].state; - d1->event_channel[port1].state = ECS_INTERDOMAIN; - port2 = get_free_port(v2); - d1->event_channel[port1].state = tmp; + u16 state = chn1->state; + chn1->state = ECS_INTERDOMAIN; + port2 = get_free_port(d2); + chn1->state = state; if ( port2 < 0 ) ERROR_EXIT(port2); } - else if ( port2 >= d2->max_event_channel ) + else if ( !port_is_valid(d2, port2) ) ERROR_EXIT(-EINVAL); + chn2 = evtchn_from_port(d2, port2); /* Validate 's current state. */ - switch ( d1->event_channel[port1].state ) + switch ( chn1->state ) { case ECS_FREE: break; case ECS_UNBOUND: - if ( d1->event_channel[port1].u.unbound.remote_domid != dom2 ) + if ( chn1->u.unbound.remote_domid != dom2 ) ERROR_EXIT(-EINVAL); break; case ECS_INTERDOMAIN: - if ( d1->event_channel[port1].u.interdomain.remote_dom != v2 ) + if ( chn1->u.interdomain.remote_dom != d2 ) ERROR_EXIT(-EINVAL); - if ( (d1->event_channel[port1].u.interdomain.remote_port != port2) && - (bind->port2 != 0) ) + if ( (chn1->u.interdomain.remote_port != port2) && (bind->port2 != 0) ) ERROR_EXIT(-EINVAL); - port2 = d1->event_channel[port1].u.interdomain.remote_port; + port2 = chn1->u.interdomain.remote_port; goto out; default: @@ -191,7 +169,7 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) } /* Validate 's current state. */ - switch ( d2->event_channel[port2].state ) + switch ( chn2->state ) { case ECS_FREE: if ( !IS_PRIV(current->domain) && (dom2 != DOMID_SELF) ) @@ -199,17 +177,16 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) break; case ECS_UNBOUND: - if ( d2->event_channel[port2].u.unbound.remote_domid != dom1 ) + if ( chn2->u.unbound.remote_domid != dom1 ) ERROR_EXIT(-EINVAL); break; case ECS_INTERDOMAIN: - if ( d2->event_channel[port2].u.interdomain.remote_dom != v1 ) + if ( chn2->u.interdomain.remote_dom != d1 ) ERROR_EXIT(-EINVAL); - if ( (d2->event_channel[port2].u.interdomain.remote_port != port1) && - (bind->port1 != 0) ) + if ( (chn2->u.interdomain.remote_port != port1) && (bind->port1 != 0) ) ERROR_EXIT(-EINVAL); - port1 = d2->event_channel[port2].u.interdomain.remote_port; + port1 = chn2->u.interdomain.remote_port; goto out; default: @@ -220,18 +197,18 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) * Everything checked out okay -- bind to . */ - d1->event_channel[port1].u.interdomain.remote_dom = v2; - d1->event_channel[port1].u.interdomain.remote_port = (u16)port2; - d1->event_channel[port1].state = ECS_INTERDOMAIN; + chn1->u.interdomain.remote_dom = d2; + chn1->u.interdomain.remote_port = (u16)port2; + chn1->state = ECS_INTERDOMAIN; - d2->event_channel[port2].u.interdomain.remote_dom = v1; - d2->event_channel[port2].u.interdomain.remote_port = (u16)port1; - d2->event_channel[port2].state = ECS_INTERDOMAIN; + chn2->u.interdomain.remote_dom = d1; + chn2->u.interdomain.remote_port = (u16)port1; + chn2->state = ECS_INTERDOMAIN; out: - spin_unlock(&d1->event_channel_lock); + spin_unlock(&d1->evtchn_lock); if ( d1 != d2 ) - spin_unlock(&d2->event_channel_lock); + spin_unlock(&d2->evtchn_lock); put_domain(d1); put_domain(d2); @@ -246,6 +223,7 @@ static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind) static long evtchn_bind_virq(evtchn_bind_virq_t *bind) { + struct evtchn *chn; struct vcpu *v = current; struct domain *d = v->domain; int port, virq = bind->virq; @@ -253,23 +231,25 @@ static long evtchn_bind_virq(evtchn_bind_virq_t *bind) if ( virq >= ARRAY_SIZE(v->virq_to_evtchn) ) return -EINVAL; - spin_lock(&d->event_channel_lock); + spin_lock(&d->evtchn_lock); /* * Port 0 is the fallback port for VIRQs that haven't been explicitly * bound yet. */ if ( ((port = v->virq_to_evtchn[virq]) != 0) || - ((port = get_free_port(v)) < 0) ) + ((port = get_free_port(d)) < 0) ) goto out; - d->event_channel[port].state = ECS_VIRQ; - d->event_channel[port].u.virq = virq; + chn = evtchn_from_port(d, port); + chn->state = ECS_VIRQ; + chn->notify_vcpu_id = v->vcpu_id; + chn->u.virq = virq; v->virq_to_evtchn[virq] = port; out: - spin_unlock(&d->event_channel_lock); + spin_unlock(&d->evtchn_lock); if ( port < 0 ) return port; @@ -278,24 +258,26 @@ static long evtchn_bind_virq(evtchn_bind_virq_t *bind) return 0; } + static long evtchn_bind_ipi(evtchn_bind_ipi_t *bind) { - struct vcpu *v = current; - struct domain *d = v->domain; + struct evtchn *chn; + struct domain *d = current->domain; int port, ipi_vcpu = bind->ipi_vcpu; - if ( ipi_vcpu >= MAX_VIRT_CPUS ) + if ( (ipi_vcpu >= MAX_VIRT_CPUS) || (d->vcpu[ipi_vcpu] == NULL) ) return -EINVAL; - spin_lock(&d->event_channel_lock); + spin_lock(&d->evtchn_lock); - if ( (port = get_free_port(v)) >= 0 ) + if ( (port = get_free_port(d)) >= 0 ) { - d->event_channel[port].state = ECS_IPI; - d->event_channel[port].u.ipi_vcpu = ipi_vcpu; + chn = evtchn_from_port(d, port); + chn->state = ECS_IPI; + chn->notify_vcpu_id = ipi_vcpu; } - spin_unlock(&d->event_channel_lock); + spin_unlock(&d->evtchn_lock); if ( port < 0 ) return port; @@ -307,20 +289,23 @@ static long evtchn_bind_ipi(evtchn_bind_ipi_t *bind) static long evtchn_bind_pirq(evtchn_bind_pirq_t *bind) { + struct evtchn *chn; struct domain *d = current->domain; int port, rc, pirq = bind->pirq; if ( pirq >= ARRAY_SIZE(d->pirq_to_evtchn) ) return -EINVAL; - spin_lock(&d->event_channel_lock); + spin_lock(&d->evtchn_lock); if ( ((rc = port = d->pirq_to_evtchn[pirq]) != 0) || - ((rc = port = get_free_port(current)) < 0) ) + ((rc = port = get_free_port(d)) < 0) ) goto out; + chn = evtchn_from_port(d, port); + d->pirq_to_evtchn[pirq] = port; - rc = pirq_guest_bind(current, pirq, + rc = pirq_guest_bind(d->vcpu[chn->notify_vcpu_id], pirq, !!(bind->flags & BIND_PIRQ__WILL_SHARE)); if ( rc != 0 ) { @@ -328,11 +313,11 @@ static long evtchn_bind_pirq(evtchn_bind_pirq_t *bind) goto out; } - d->event_channel[port].state = ECS_PIRQ; - d->event_channel[port].u.pirq = pirq; + chn->state = ECS_PIRQ; + chn->u.pirq = pirq; out: - spin_unlock(&d->event_channel_lock); + spin_unlock(&d->evtchn_lock); if ( rc < 0 ) return rc; @@ -344,24 +329,23 @@ static long evtchn_bind_pirq(evtchn_bind_pirq_t *bind) static long __evtchn_close(struct domain *d1, int port1) { - struct domain *d2 = NULL; - struct vcpu *v; - event_channel_t *chn1, *chn2; - int port2; - long rc = 0; + struct domain *d2 = NULL; + struct vcpu *v; + struct evtchn *chn1, *chn2; + int port2; + long rc = 0; again: - spin_lock(&d1->event_channel_lock); - - chn1 = d1->event_channel; + spin_lock(&d1->evtchn_lock); - if ( (port1 < 0) || (port1 >= d1->max_event_channel) ) + if ( !port_is_valid(d1, port1) ) { rc = -EINVAL; goto out; } - switch ( chn1[port1].state ) + chn1 = evtchn_from_port(d1, port1); + switch ( chn1->state ) { case ECS_FREE: case ECS_RESERVED: @@ -372,15 +356,14 @@ static long __evtchn_close(struct domain *d1, int port1) break; case ECS_PIRQ: - if ( (rc = pirq_guest_unbind(d1, chn1[port1].u.pirq)) == 0 ) - d1->pirq_to_evtchn[chn1[port1].u.pirq] = 0; + if ( (rc = pirq_guest_unbind(d1, chn1->u.pirq)) == 0 ) + d1->pirq_to_evtchn[chn1->u.pirq] = 0; break; case ECS_VIRQ: - /* XXX could store vcpu in chn1[port1].u */ for_each_vcpu ( d1, v ) - if (v->virq_to_evtchn[chn1[port1].u.virq] == port1) - v->virq_to_evtchn[chn1[port1].u.virq] = 0; + if ( v->virq_to_evtchn[chn1->u.virq] == port1 ) + v->virq_to_evtchn[chn1->u.virq] = 0; break; case ECS_IPI: @@ -389,7 +372,7 @@ static long __evtchn_close(struct domain *d1, int port1) case ECS_INTERDOMAIN: if ( d2 == NULL ) { - d2 = chn1[port1].u.interdomain.remote_dom->domain; + d2 = chn1->u.interdomain.remote_dom; /* If we unlock d1 then we could lose d2. Must get a reference. */ if ( unlikely(!get_domain(d2)) ) @@ -404,50 +387,47 @@ static long __evtchn_close(struct domain *d1, int port1) if ( d1 < d2 ) { - spin_lock(&d2->event_channel_lock); + spin_lock(&d2->evtchn_lock); } else if ( d1 != d2 ) { - spin_unlock(&d1->event_channel_lock); - spin_lock(&d2->event_channel_lock); + spin_unlock(&d1->evtchn_lock); + spin_lock(&d2->evtchn_lock); goto again; } } - else if ( d2 != chn1[port1].u.interdomain.remote_dom->domain ) + else if ( d2 != chn1->u.interdomain.remote_dom ) { rc = -EINVAL; goto out; } - chn2 = d2->event_channel; - port2 = chn1[port1].u.interdomain.remote_port; - - if ( port2 >= d2->max_event_channel ) - BUG(); - if ( chn2[port2].state != ECS_INTERDOMAIN ) - BUG(); - if ( chn2[port2].u.interdomain.remote_dom->domain != d1 ) - BUG(); - - chn2[port2].state = ECS_UNBOUND; - chn2[port2].u.unbound.remote_domid = d1->domain_id; + port2 = chn1->u.interdomain.remote_port; + BUG_ON(!port_is_valid(d2, port2)); + + chn2 = evtchn_from_port(d2, port2); + BUG_ON(chn2->state != ECS_INTERDOMAIN); + BUG_ON(chn2->u.interdomain.remote_dom != d1); + + chn2->state = ECS_UNBOUND; + chn2->u.unbound.remote_domid = d1->domain_id; break; default: BUG(); } - chn1[port1].state = ECS_FREE; + chn1->state = ECS_FREE; out: if ( d2 != NULL ) { if ( d1 != d2 ) - spin_unlock(&d2->event_channel_lock); + spin_unlock(&d2->evtchn_lock); put_domain(d2); } - spin_unlock(&d1->event_channel_lock); + spin_unlock(&d1->evtchn_lock); return rc; } @@ -476,50 +456,52 @@ static long evtchn_close(evtchn_close_t *close) long evtchn_send(int lport) { - struct domain *ld = current->domain; - struct vcpu *rd; + struct evtchn *lchn, *rchn; + struct domain *ld = current->domain, *rd; int rport, ret = 0; - spin_lock(&ld->event_channel_lock); + spin_lock(&ld->evtchn_lock); - if ( unlikely(lport < 0) || - unlikely(lport >= ld->max_event_channel)) + if ( unlikely(!port_is_valid(ld, lport)) ) { - spin_unlock(&ld->event_channel_lock); + spin_unlock(&ld->evtchn_lock); return -EINVAL; } - switch ( ld->event_channel[lport].state ) + lchn = evtchn_from_port(ld, lport); + switch ( lchn->state ) { case ECS_INTERDOMAIN: - rd = ld->event_channel[lport].u.interdomain.remote_dom; - rport = ld->event_channel[lport].u.interdomain.remote_port; - - evtchn_set_pending(rd, rport); + rd = lchn->u.interdomain.remote_dom; + rport = lchn->u.interdomain.remote_port; + rchn = evtchn_from_port(rd, rport); + evtchn_set_pending(rd->vcpu[rchn->notify_vcpu_id], rport); break; case ECS_IPI: - rd = ld->vcpu[ld->event_channel[lport].u.ipi_vcpu]; - if ( rd ) - evtchn_set_pending(rd, lport); - else - ret = -EINVAL; + evtchn_set_pending(ld->vcpu[lchn->notify_vcpu_id], lport); break; default: ret = -EINVAL; } - spin_unlock(&ld->event_channel_lock); + spin_unlock(&ld->evtchn_lock); return ret; } +void send_guest_pirq(struct domain *d, int pirq) +{ + int port = d->pirq_to_evtchn[pirq]; + struct evtchn *chn = evtchn_from_port(d, port); + evtchn_set_pending(d->vcpu[chn->notify_vcpu_id], port); +} static long evtchn_status(evtchn_status_t *status) { struct domain *d; domid_t dom = status->dom; int port = status->port; - event_channel_t *chn; + struct evtchn *chn; long rc = 0; if ( dom == DOMID_SELF ) @@ -530,17 +512,16 @@ static long evtchn_status(evtchn_status_t *status) if ( (d = find_domain_by_id(dom)) == NULL ) return -ESRCH; - spin_lock(&d->event_channel_lock); - - chn = d->event_channel; + spin_lock(&d->evtchn_lock); - if ( (port < 0) || (port >= d->max_event_channel) ) + if ( !port_is_valid(d, port) ) { rc = -EINVAL; goto out; } - switch ( chn[port].state ) + chn = evtchn_from_port(d, port); + switch ( chn->state ) { case ECS_FREE: case ECS_RESERVED: @@ -548,32 +529,32 @@ static long evtchn_status(evtchn_status_t *status) break; case ECS_UNBOUND: status->status = EVTCHNSTAT_unbound; - status->u.unbound.dom = chn[port].u.unbound.remote_domid; + status->u.unbound.dom = chn->u.unbound.remote_domid; break; case ECS_INTERDOMAIN: status->status = EVTCHNSTAT_interdomain; status->u.interdomain.dom = - chn[port].u.interdomain.remote_dom->domain->domain_id; - status->u.interdomain.port = chn[port].u.interdomain.remote_port; + chn->u.interdomain.remote_dom->domain_id; + status->u.interdomain.port = chn->u.interdomain.remote_port; break; case ECS_PIRQ: status->status = EVTCHNSTAT_pirq; - status->u.pirq = chn[port].u.pirq; + status->u.pirq = chn->u.pirq; break; case ECS_VIRQ: status->status = EVTCHNSTAT_virq; - status->u.virq = chn[port].u.virq; + status->u.virq = chn->u.virq; break; case ECS_IPI: status->status = EVTCHNSTAT_ipi; - status->u.ipi_vcpu = chn[port].u.ipi_vcpu; + status->u.ipi_vcpu = chn->notify_vcpu_id; break; default: BUG(); } out: - spin_unlock(&d->event_channel_lock); + spin_unlock(&d->evtchn_lock); put_domain(d); return rc; } @@ -642,26 +623,26 @@ long do_event_channel_op(evtchn_op_t *uop) } -int init_event_channels(struct domain *d) +int evtchn_init(struct domain *d) { - spin_lock_init(&d->event_channel_lock); - /* Call get_free_port to initialize d->event_channel */ - if ( get_free_port(d->vcpu[0]) != 0 ) + spin_lock_init(&d->evtchn_lock); + if ( get_free_port(d) != 0 ) return -EINVAL; - d->event_channel[0].state = ECS_RESERVED; + evtchn_from_port(d, 0)->state = ECS_RESERVED; return 0; } -void destroy_event_channels(struct domain *d) +void evtchn_destroy(struct domain *d) { int i; - if ( d->event_channel != NULL ) - { - for ( i = 0; i < d->max_event_channel; i++ ) + + for ( i = 0; port_is_valid(d, i); i++ ) (void)__evtchn_close(d, i); - xfree(d->event_channel); - } + + for ( i = 0; i < NR_EVTCHN_BUCKETS; i++ ) + if ( d->evtchn[i] != NULL ) + xfree(d->evtchn[i]); } /* diff --git a/xen/include/public/event_channel.h b/xen/include/public/event_channel.h index a1973c6ea7..b00cda66c4 100644 --- a/xen/include/public/event_channel.h +++ b/xen/include/public/event_channel.h @@ -51,9 +51,11 @@ typedef struct { } PACKED evtchn_bind_interdomain_t; /* 12 bytes */ /* - * EVTCHNOP_bind_virq: Bind a local event channel to IRQ . + * EVTCHNOP_bind_virq: Bind a local event channel to IRQ on calling vcpu. * NOTES: - * 1. A virtual IRQ may be bound to at most one event channel per domain. + * 1. A virtual IRQ may be bound to at most one event channel per vcpu. + * 2. The allocated event channel is bound to the calling vcpu. The binding + * may not be changed. */ #define EVTCHNOP_bind_virq 1 typedef struct { @@ -79,6 +81,20 @@ typedef struct { u32 port; /* 8 */ } PACKED evtchn_bind_pirq_t; /* 12 bytes */ +/* + * EVTCHNOP_bind_ipi: Bind a local event channel to receive events. + * NOTES: + * 1. The allocated event channel is bound to the calling vcpu. The binding + * may not be changed. + */ +#define EVTCHNOP_bind_ipi 7 +typedef struct { + /* IN parameters. */ + u32 ipi_vcpu; /* 0 */ + /* OUT parameters. */ + u32 port; /* 4 */ +} PACKED evtchn_bind_ipi_t; /* 8 bytes */ + /* * EVTCHNOP_close: Close the communication channel which has an endpoint at * . If the channel is interdomain then the remote end is placed in @@ -145,18 +161,6 @@ typedef struct { } PACKED u; } PACKED evtchn_status_t; /* 20 bytes */ -/* - * EVTCHNOP_bind_ipi: Bind a local event channel to receive events. - */ -#define EVTCHNOP_bind_ipi 7 -typedef struct { - /* IN parameters. */ - u32 ipi_vcpu; /* 0 */ - /* OUT parameters. */ - u32 port; /* 4 */ -} PACKED evtchn_bind_ipi_t; /* 8 bytes */ - - typedef struct { u32 cmd; /* EVTCHNOP_* */ /* 0 */ u32 __reserved; /* 4 */ @@ -165,10 +169,10 @@ typedef struct { evtchn_bind_interdomain_t bind_interdomain; evtchn_bind_virq_t bind_virq; evtchn_bind_pirq_t bind_pirq; + evtchn_bind_ipi_t bind_ipi; evtchn_close_t close; evtchn_send_t send; evtchn_status_t status; - evtchn_bind_ipi_t bind_ipi; u8 __dummy[24]; } PACKED u; } PACKED evtchn_op_t; /* 32 bytes */ diff --git a/xen/include/xen/event.h b/xen/include/xen/event.h index a7d911771d..734427266b 100644 --- a/xen/include/xen/event.h +++ b/xen/include/xen/event.h @@ -53,7 +53,7 @@ static inline void evtchn_set_pending(struct vcpu *v, int port) /* * send_guest_virq: - * @d: Domain to which virtual IRQ should be sent + * @v: VCPU to which virtual IRQ should be sent * @virq: Virtual IRQ number (VIRQ_*) */ static inline void send_guest_virq(struct vcpu *v, int virq) @@ -69,10 +69,7 @@ static inline void send_guest_virq(struct vcpu *v, int virq) * @d: Domain to which physical IRQ should be sent * @pirq: Physical IRQ number */ -static inline void send_guest_pirq(struct vcpu *v, int pirq) -{ - evtchn_set_pending(v, v->domain->pirq_to_evtchn[pirq]); -} +extern void send_guest_pirq(struct domain *d, int pirq); #define event_pending(_d) \ ((_d)->vcpu_info->evtchn_upcall_pending && \ diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h index 3eb2537fa5..1237da7909 100644 --- a/xen/include/xen/sched.h +++ b/xen/include/xen/sched.h @@ -19,7 +19,11 @@ extern rwlock_t domlist_lock; /* A global pointer to the initial domain (DOM0). */ extern struct domain *dom0; -typedef struct event_channel_st +#define MAX_EVTCHNS 1024 +#define EVTCHNS_PER_BUCKET 128 +#define NR_EVTCHN_BUCKETS (MAX_EVTCHNS / EVTCHNS_PER_BUCKET) + +struct evtchn { #define ECS_FREE 0 /* Channel is available for use. */ #define ECS_RESERVED 1 /* Channel is reserved. */ @@ -28,24 +32,23 @@ typedef struct event_channel_st #define ECS_PIRQ 4 /* Channel is bound to a physical IRQ line. */ #define ECS_VIRQ 5 /* Channel is bound to a virtual IRQ line. */ #define ECS_IPI 6 /* Channel is bound to a virtual IPI line. */ - u16 state; + u16 state; /* ECS_* */ + u16 notify_vcpu_id; /* VCPU for local delivery notification */ union { struct { domid_t remote_domid; - } __attribute__ ((packed)) unbound; /* state == ECS_UNBOUND */ + } unbound; /* state == ECS_UNBOUND */ struct { - u16 remote_port; - struct vcpu *remote_dom; - } __attribute__ ((packed)) interdomain; /* state == ECS_INTERDOMAIN */ - u16 pirq; /* state == ECS_PIRQ */ - u16 virq; /* state == ECS_VIRQ */ - u32 ipi_vcpu; /* state == ECS_IPI */ + u16 remote_port; + struct domain *remote_dom; + } interdomain; /* state == ECS_INTERDOMAIN */ + u16 pirq; /* state == ECS_PIRQ */ + u16 virq; /* state == ECS_VIRQ */ } u; -} event_channel_t; +}; -int init_event_channels(struct domain *d); -void destroy_event_channels(struct domain *d); -int init_vcpu_event_channels(struct vcpu *v); +int evtchn_init(struct domain *d); +void evtchn_destroy(struct domain *d); #define CPUMAP_RUNANYWHERE 0xFFFFFFFF @@ -109,9 +112,8 @@ struct domain struct domain *next_in_hashbucket; /* Event channel information. */ - event_channel_t *event_channel; - unsigned int max_event_channel; - spinlock_t event_channel_lock; + struct evtchn *evtchn[NR_EVTCHN_BUCKETS]; + spinlock_t evtchn_lock; grant_table_t *grant_table; -- 2.30.2